'use strict'
//爬取数据将模型与数据库同步
var Job = require('../models/job');
var Crawler = require('crawler');

var crawler = new Crawler({
    maxConnections: 10,
    //将为每个抓取的页面调用此回调函数
    callback: function (error, res, done) {
        if (error) {
            console.log(`爬取错误${error}`)
        } else {
            var $ = res.$; //cheerio
            console.log($('title').text());
            var data = $('.dw_table .el').toArray(); //51
            data.forEach(element => {
                var jobName = $(element).find('.t1 a').attr('title');
                if (jobName == '数据分析师') {
                    var companyName = $(element).find('.t2 a').text().trim().replace(/\s/g, '');
                    var workPlace = $(element).find('.t3').text().trim().replace(/\s/g, '');
                    var pay = $(element).find('.t4').text().trim().replace(/\s/g, '');
                    var workTime = $(element).find('.t5').text().trim().replace(/\s/g, '');
                    console.log(jobName, companyName, workPlace, pay, workTime);
                    Job.sync({
                        force: true
                    }).then(() => {
                        Job.create({
                            JobName: jobName,
                            CompanyName: companyName,
                            WorkPlace: workPlace,
                            Pay: pay,
                            WorkTime: workTime
                        })
                    })
                }

            });
        }
        done(); //爬取完成

    }
})

// crawler.queue('https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=');
// module.exports = crawler